#include "../include/DictProducer.hpp"
#include "../include/Configuration.hpp"
#include "../include/CppJieba.hpp"
#include "../include/DirScanner.hpp"

#include <fstream>
#include <iostream>
#include <sstream>
#include <ostream>
#include <memory>

using std::endl;
using std::cout;
using std::ifstream;
using std::ofstream;
using std::istringstream;
using std::shared_ptr;

class DirScanner;
//构建英文字典库
void DictProducer::buildEnDict()
{
    Configuration* conf = Configuration::getInstance();
    //调用getinstance，创建多个指针，但是都指向同一个单例类Conf
    map<string, string> mp = conf->getConfigMap();
    set<string> st = conf->getStopWordList();
    
    string dic = mp["enyuliao"];
    shared_ptr<DirScanner> scan(new DirScanner());
    scan->operator()(dic);
    vector<string> files = scan->files();

    ifstream ifs;
    for(auto it = files.begin(); it != files.end(); ++it)
    {
        string filename = *it;
        ifs.open(filename, std::ios::in);
        if(!ifs)
        {
            cout << "DictProducer::buildEnDict ifstream open " << filename << "error" << endl;
        }

        //需要将停用词先读出来放入进容器//是否需要写一个函数去读取停用词文件
        string lineData;
        size_t lineNumber = 0;//行号

        while(std::getline(ifs, lineData), ifs.good())
        {
            //过滤改行数据,变为小写模式,分割单词
            preProcessLine(lineData);


            istringstream line(lineData);
            string word;
            while(line >>  word)
            {
                //查看是否是停用词
                auto sit = _stop.find(word);//时间复杂度O(1)
                if(sit != _stop.end())
                {
                    continue;//跳出进入下一个单词
                }
                //索引库的建立 此时需要将单词分解   索引库是否应该将停用词剔除?
                createEnIndex(word,lineNumber);

                //词库的建立
                auto it = _dict.find(word);
            
                if(it != _dict.end())//找到了
                {
                    ++_dict[word];//词频++
                }
                else
                {
                    _dict[word] = 1;//插入且词频置为1
                }
            }
            ++lineNumber;
        }
        ifs.close();

    }
    
}

void DictProducer::preProcessLine(string &line)
{
    for(size_t i=0;i< line.size();++i)
    {
        if(!isalpha(line[i]))//处理单词，如果不是字母就用空格代替abc? abc123 Hello
        {
            line[i] = ' ';
        }
        else if(isupper(line[i]))//如果是大写就转为小写
        {
            line[i] = tolower(line[i]);
        }
    }
}

void DictProducer::buildCnDict()
{
    CppJieba jieba;
    _cuttor = &jieba;
    //基类指针指向派生类对象
    //
    Configuration* conf = Configuration::getInstance();
    //调用getinstance，创建多个指针，但是都指向同一个单例类Conf        
    map<string, string> mp = conf->getConfigMap();
    set<string> st = conf->getStopWordList();
    string dic = mp["cnyuliao"];
    shared_ptr<DirScanner> scan(new DirScanner());
    scan->operator()(dic);
    vector<string> files = scan->files();
    
    ifstream ifs;
    for(auto it = files.begin(); it != files.end(); ++it)
    {
        string filename = *it;
        ifs.open(filename, std::ios::in);
        if(!ifs)
        {
            cout << "DictProducer::buildCnDict ifstream open " << filename << "error" << endl;
        }

        string lineData;
        size_t lineNumber = 0;//行号
        while(std::getline(ifs, lineData), ifs.good())
        {

            lineData = censorStrWithOutSymbol(lineData);

           vector<string>tmp = _cuttor->cut(lineData);

           //遍历tmp看是否有与停用词重复的
           for(auto &it : tmp)
           {
               auto sit = _stop.find(it);
               if(sit != _stop.end())
               {
                   continue;
               }

               createCnIndex(it, lineNumber);

               auto it1 = _cndict.find(it);
               if(it1 != _cndict.end())
               {
                   ++_cndict[it];//词频
               }else
               {
                   _cndict[it] = 1;
               }
           }
           ++lineNumber;
        }
        ifs.close();
    }
}

void DictProducer::createEnIndex(const string & word,int lineNum)//英文索引库
{
    for(size_t i=0; i < word.size(); ++i)
    {
        _index[word[i]].insert(lineNum);
    }
}

void DictProducer::createCnIndex(const string &word, int lineNum)
{
    string tmp = word;
    vector<string> vec;//切分后的单个字

    int size = 3;
    for(size_t i = 0; i < tmp.size(); i += size)
    {
        string tmp1 = tmp.substr(i, size);
        _cnIndex[tmp1].insert(lineNum);
    }
}

//该函数需要指定持久化文件的绝对路径作为第一个参数才行
//
void DictProducer::enStore(const string &dict, const string &index)
{
    //建立词典库
    ofstream ofs(dict, std::ios::out | std::ios::trunc);
    for(auto &it : _dict)
    {
        ofs << it.first << ' ' << it.second << endl;
    }
    ofs.close();

    ofstream ofs2(index);
    for(auto &it1 : _index)
    {
        ofs2 << it1.first << " ";
        set<int>::iterator it3 = it1.second.begin();
        for(;it3 != it1.second.end(); ++it3)
        {
            ofs2 << *it3 << " ";
        }
        ofs2 << endl;
    }
    ofs2.close();
}

//同理也需要指定持久化文件路径，暂定为存到./data/lib里
//
void DictProducer::cnStore(const string &dict, const string &index)
{
    //建立词典库
    ofstream ofs(dict);
    for(auto &it : _cndict)
    {
        ofs << it.first << ' ' << it.second << endl;
    }
    ofs.close();

    ofstream ofs2(index);
    for(auto &it1 : _cnIndex)
    {
        ofs2 << it1.first << " ";
        set<int>::iterator it3 = it1.second.begin();
        for(;it3 != it1.second.end(); ++it3)
        {
            ofs2 << *it3 << " ";
        }
        ofs2 << endl;
    }
    ofs.close();
}


string DictProducer::censorStrWithOutSymbol(const string &source)
{
    string sourceWithOutSymbol;
    // regex reg("[\u4e00-\u9fa5]+");

    // auto word_begin=std::sregex_iterator(source.begin(), source.end(), reg);//正则匹配中文
	// auto word_end=std::sregex_iterator();

    // for(auto it=word_begin;it!=word_end;++it)
    // {
    //     sourceWithOutSymbol+=(*it).str();
    // }

    int i = 0;
    while (source[i] != 0)
    {
        if (source[i] & 0x80 && source[i] & 0x40 && source[i] & 0x20)
        {
            int byteCount = 0;
            if (source[i] & 0x10)
            {
                byteCount = 4;
            }
            else
            {
                byteCount = 3;
            }
            for (int a = 0; a < byteCount; a++)
            {
                sourceWithOutSymbol += source[i];
                i++;
            }
        }
        else if (source[i] & 0x80 && source[i] & 0x40)
        {
            i += 2;
        }
        else
        {
            i += 1;
        }
    }
    auto it = std::remove_if(sourceWithOutSymbol.begin(), sourceWithOutSymbol.end(), ::ispunct);
    sourceWithOutSymbol.erase(it, sourceWithOutSymbol.end());
    return sourceWithOutSymbol;
}
